{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support.ui import Select\n",
    "\n",
    "import chromedriver_binary  # Adds chromedriver binary to path\n",
    "\n",
    "import time,os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import random\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HELPER FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Quits the browser\n",
    "def quitBrowser():\n",
    "    try:\n",
    "        browser.quit()\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "\n",
    "# Returns a new browser\n",
    "def newBrowser():\n",
    "\n",
    "    browserProfile = webdriver.ChromeOptions()\n",
    "    browserProfile.add_experimental_option(\n",
    "        'prefs', {'intl.accept_languages': 'en,en_US'})\n",
    "\n",
    "    browser = webdriver.Chrome(options=browserProfile)\n",
    "    \n",
    "    # In case chromedriver_binary does not work, use this:\n",
    "    # path = XXX\n",
    "    # browser = webdriver.Chrome(path, options=browserProfile)\n",
    "\n",
    "    # If an element is not found, browser will try again every 0.5s until 3 seconds\n",
    "    browser.implicitly_wait(3)\n",
    "\n",
    "    return browser\n",
    "\n",
    "\n",
    "# Logs in to AWS using the credentials in AWS_credentials.txt\n",
    "def awsLogin():\n",
    "\n",
    "    with open(\"AWS_credentials.txt\", 'r') as f:\n",
    "        [aws_id, username, password] = f.read().splitlines()\n",
    "\n",
    "    # Build AWS Console URL with aws_id\n",
    "    aws_id = str(aws_id)\n",
    "    url = \"https://%s.signin.aws.amazon.com/console\" % aws_id\n",
    "\n",
    "    # Open browser with the starting URL\n",
    "    browser.get(url)\n",
    "    browser.refresh()\n",
    "    time.sleep(3)\n",
    "\n",
    "    usernameInput = browser.find_elements_by_css_selector('form input')[1]\n",
    "    passwordInput = browser.find_elements_by_css_selector('form input')[2]\n",
    "\n",
    "    usernameInput.send_keys(username)\n",
    "    passwordInput.send_keys(password)\n",
    "    passwordInput.send_keys(Keys.ENTER)\n",
    "    time.sleep(2.5)\n",
    "\n",
    "    print(\n",
    "        f\"Successfully logged in to AWS account number {aws_id} with username {username}\")\n",
    "\n",
    "\n",
    "# General function to create a new model (used by cloneModel() and newModel())\n",
    "# Note: rfchanges is not yet implemented (make changes to parameters in the reward function)\n",
    "def createModel(modelname, description, track, hyperparams, maxtime,\n",
    "                car=None, rewardfunction=None, rfchanges=None, testmode=False):\n",
    "\n",
    "    #### PAGE 1 ####\n",
    "\n",
    "    # Fill in the model name\n",
    "    time.sleep(0.5)\n",
    "    #modelnameInput = browser.find_element_by_id('awsui-input-1')\n",
    "    # awsui-input awsui-input-type-text\n",
    "    modelnameInput = browser.find_element_by_css_selector(\n",
    "        'input[class=\"awsui-input awsui-input-type-text\"]')\n",
    "    modelnameInput.clear()\n",
    "    modelnameInput.send_keys(modelname)\n",
    "\n",
    "    # Fill in the training job description\n",
    "    time.sleep(0.5)\n",
    "    descriptionInput = browser.find_element_by_css_selector(\n",
    "        'textarea[class=\"awsui-textarea\"]')\n",
    "    descriptionInput.clear()\n",
    "    descriptionInput.send_keys(description)\n",
    "\n",
    "    # Select the track\n",
    "    time.sleep(0.5)\n",
    "    trackValue = \"arn:aws:deepracer:us-east-1::track/%s_track\" % track\n",
    "    trackButton = browser.find_element_by_css_selector(\n",
    "        'input[type=\"radio\"][value=\"%s\"]' % trackValue)\n",
    "    trackButton.click()\n",
    "\n",
    "    # Click next (check page 3, part 3 for more robust xpath approach)\n",
    "    time.sleep(0.5)\n",
    "    nextButton = browser.find_elements_by_css_selector(\n",
    "        'button[type=\"submit\"] span[awsui-button-region=\"text\"]')[3]\n",
    "    nextButton.click()\n",
    "\n",
    "    #### PAGE 2 ####\n",
    "\n",
    "    # Click time trial (race type)\n",
    "    time.sleep(0.5)\n",
    "    raceType = \"TIME_TRIAL\"  # add to function parameters if needed\n",
    "    raceTypeButton = browser.find_element_by_css_selector(\n",
    "        'input[type=\"radio\"][value=\"%s\"]' % raceType)\n",
    "    raceTypeButton.click()\n",
    "\n",
    "    # Only select car if car argument is passed, else skip this part\n",
    "    if car != None:\n",
    "\n",
    "        time.sleep(0.5)\n",
    "\n",
    "        # Expand car list\n",
    "        allCarsList = browser.find_element_by_css_selector(\n",
    "            'span[class=\"awsui-select-trigger-textbox\"]')\n",
    "        allCarsList.click()\n",
    "\n",
    "        # Select desired car\n",
    "        # number of retries with 1 sec wait inbetween (expanding car list takes time)\n",
    "        retry = 10\n",
    "        while retry > 0:\n",
    "            time.sleep(1)\n",
    "            try:\n",
    "                carButton = browser.find_element_by_css_selector(\n",
    "                    'div[title=\"%s\"]' % car)\n",
    "            except:\n",
    "                retry -= 1  # this is executed when there was an error\n",
    "            else:\n",
    "                retry = 0  # this is executed when there was no error\n",
    "        carButton.click()\n",
    "\n",
    "    # Click next (check page 3, part 3 for more robust xpath approach)\n",
    "    time.sleep(0.5)\n",
    "    nextButton2 = browser.find_elements_by_css_selector(\n",
    "        'button[type=\"submit\"] span[awsui-button-region=\"text\"]')[1]\n",
    "    nextButton2.click()\n",
    "\n",
    "    #### PAGE 3, PART 1 REWARD FUNCTION ####\n",
    "\n",
    "    # Only fill out reward function if argument is passed, else leave pre-filled reward function\n",
    "    if rewardfunction != None:\n",
    "\n",
    "        time.sleep(0.5)\n",
    "\n",
    "        # Click into code editor for reward function\n",
    "        codeEditor = browser.find_elements_by_css_selector(\n",
    "            'span[class=\"ace_keyword\"]')[0]\n",
    "        actionChain1 = webdriver.ActionChains(browser)\n",
    "        actionChain1.move_to_element(codeEditor)\n",
    "        actionChain1.click()\n",
    "        actionChain1.perform()\n",
    "\n",
    "        # Select all code and delete\n",
    "        actionChain2 = webdriver.ActionChains(browser)\n",
    "        # use Keys.CONTROL for a Windows system\n",
    "        actionChain2.key_down(Keys.COMMAND)\n",
    "        actionChain2.send_keys('a')\n",
    "        actionChain2.key_up(Keys.COMMAND)\n",
    "        actionChain2.send_keys(Keys.DELETE)\n",
    "        actionChain2.perform()\n",
    "\n",
    "        # Insert reward function\n",
    "        # Note: COMMAND+C / COMMAND+V does not work with Selenium as Chrome blocks it because of security reasons\n",
    "        # NOT YET IMPLEMENTED: WOULD BE EASIER TO INSERT reward_function.txt to S3 with Boto3\n",
    "        # reward_function_to_console(rewardfunction)\n",
    "\n",
    "    #### PAGE 3, PART 2 HYPERPARAMETERS ####\n",
    "\n",
    "    # Expand hyperparameter menu\n",
    "    time.sleep(0.5)\n",
    "    hyperparamsExpand = browser.find_element_by_css_selector(\n",
    "        'awsui-expandable-section[class=\"algorithm-settings\"]')\n",
    "    hyperparamsExpand.click()\n",
    "\n",
    "    # Select desired batch size\n",
    "    time.sleep(0.5)\n",
    "    batchsizeButton = browser.find_element_by_css_selector(\n",
    "        'input[type=\"radio\"][value=\"%i\"]' % hyperparams[\"batchsize\"])\n",
    "    batchsizeButton.click()\n",
    "\n",
    "    # Enter desired number of epochs\n",
    "    time.sleep(0.5)\n",
    "    epochsField = browser.find_element_by_css_selector(\n",
    "        'input[name=\"request.TrainingConfig.Hyperparameters.num_epochs\"]')\n",
    "    epochsField.clear()\n",
    "    epochsField.send_keys(hyperparams[\"epochs\"])\n",
    "\n",
    "    # Enter desired learning rate\n",
    "    time.sleep(0.5)\n",
    "    learningrateField = browser.find_element_by_css_selector(\n",
    "        'input[name=\"request.TrainingConfig.Hyperparameters.lr\"]')\n",
    "    learningrateField.clear()\n",
    "    learningrateField.send_keys(str(hyperparams[\"learningrate\"]))\n",
    "\n",
    "    # Enter desired entropy\n",
    "    time.sleep(0.5)\n",
    "    entropyField = browser.find_element_by_css_selector(\n",
    "        'input[name=\"request.TrainingConfig.Hyperparameters.beta_entropy\"]')\n",
    "    entropyField.clear()\n",
    "    entropyField.send_keys(str(hyperparams[\"entropy\"]))\n",
    "\n",
    "    # Enter desired discount factor\n",
    "    time.sleep(0.5)\n",
    "    discountField = browser.find_element_by_css_selector(\n",
    "        'input[name=\"request.TrainingConfig.Hyperparameters.discount_factor\"]')\n",
    "    discountField.clear()\n",
    "    discountField.send_keys(str(hyperparams[\"discount\"]))\n",
    "\n",
    "    # Enter desired episodes between updates\n",
    "    time.sleep(0.5)\n",
    "    episodesUpdateField = browser.find_element_by_css_selector(\n",
    "        'input[name=\"request.TrainingConfig.Hyperparameters.num_episodes_between_training\"]')\n",
    "    episodesUpdateField.clear()\n",
    "    episodesUpdateField.send_keys(str(hyperparams[\"episodesUpdate\"]))\n",
    "\n",
    "    #### PAGE 3, PART 3 STOP CONDITION ####\n",
    "\n",
    "    # Enter desired maximum training time in minutes\n",
    "    time.sleep(0.5)\n",
    "    episodesUpdateField = browser.find_element_by_css_selector(\n",
    "        'input[name=\"request.TrainingConfig.TerminationConditions.MaxTimeInMinutes\"]')\n",
    "    episodesUpdateField.clear()\n",
    "    episodesUpdateField.send_keys(str(maxtime))\n",
    "\n",
    "    # Press \"Create model\", but only if testmode is False\n",
    "    if testmode == False:\n",
    "        time.sleep(0.5)\n",
    "        createModelButton = browser.find_element_by_xpath(\n",
    "            '//button[@type=\"submit\"]/*[text()=\"Create model\"]')\n",
    "        createModelButton.text\n",
    "        createModelButton.click()\n",
    "        time.sleep(15)\n",
    "        # Print success state\n",
    "        print(\n",
    "            f\"Successfully created model {modelname} with hyperparams {hyperparams}\")\n",
    "    else:\n",
    "        print(\n",
    "            f\"Prepared model with name {modelname}, but did not yet create it\")\n",
    "\n",
    "\n",
    "# Creates new model as clone from other model\n",
    "def cloneModel(clonefrom, modelname, description, track, hyperparams,\n",
    "               maxtime, rewardfunction=None, rfchanges=None, testmode=False):\n",
    "\n",
    "    browser.get(\n",
    "        \"https://console.aws.amazon.com/deepracer/home?region=us-east-1#model/%s\" % clonefrom)\n",
    "    browser.refresh()\n",
    "    time.sleep(3)\n",
    "\n",
    "    # Click on clone button\n",
    "    time.sleep(0.5)\n",
    "    cloneButton = browser.find_element_by_xpath(\n",
    "        '//*[@id=\"PLCHLDR_model_detail_clone_button\"]')\n",
    "    cloneButton.click()\n",
    "\n",
    "    # Create model as clone\n",
    "    time.sleep(0.5)\n",
    "    createModel(modelname=modelname,\n",
    "                description=description,\n",
    "                track=track,\n",
    "                hyperparams=hyperparams,\n",
    "                maxtime=maxtime,\n",
    "                rewardfunction=rewardfunction,\n",
    "                rfchanges=rfchanges,\n",
    "                testmode=testmode)\n",
    "\n",
    "\n",
    "# Counts the number of models that are currently training\n",
    "def count_models_training():\n",
    "\n",
    "    browser.get(\n",
    "        \"https://console.aws.amazon.com/deepracer/home?region=us-east-1#models\")\n",
    "    browser.refresh()\n",
    "    time.sleep(3)\n",
    "\n",
    "    # Count number of models that are being created\n",
    "    count_created = len(browser.find_elements_by_xpath(\n",
    "        '//span/*[text()=\"Created\"]'))\n",
    "    # Count number of models that are training\n",
    "    count_training = len(browser.find_elements_by_xpath(\n",
    "        '//span/*[text()=\"Training...\"]'))\n",
    "    # Count number of models that are being stopped\n",
    "    count_stopping = len(browser.find_elements_by_xpath(\n",
    "        '//span/*[text()=\"Stopping...\"]'))\n",
    "\n",
    "    return count_created + count_training + count_stopping\n",
    "\n",
    "\n",
    "# Clones model and performs multiple experiments with hyperparameters\n",
    "def clone_hyperparams_experiment(clone_from_model, hyperparams_experiment, training_slots=2,\n",
    "                                 number_of_experiments=2, start_naming_with=\"a\", track=\"reInvent2019\",\n",
    "                                 maxtime_per_training=180):\n",
    "\n",
    "    # Calculate approximate number of hours that this function will run\n",
    "    total_hours = number_of_experiments * \\\n",
    "        (maxtime_per_training/60) / training_slots\n",
    "    print(\n",
    "        f\"Starting {number_of_experiments} experiments. This will take approx {total_hours} hours.\")\n",
    "\n",
    "    # Create all combinations of hyperparameters. Result is list of dictionaries\n",
    "    keys, values = zip(*hyperparams_experiment.items())\n",
    "    hp_exp_all = [dict(zip(keys, v)) for v in itertools.product(*values)]\n",
    "\n",
    "    # Generate random indexes\n",
    "    hp_exp_all_indexes = random.sample(\n",
    "        range(len(hp_exp_all)), number_of_experiments)\n",
    "\n",
    "    # Select only the dictionaries that were randomly selected\n",
    "    hp_exp = [hp_exp_all[i] for i in hp_exp_all_indexes]\n",
    "\n",
    "    # Transform e.g. \"a\" to integer 97\n",
    "    start_naming_with_int = ord(start_naming_with)\n",
    "\n",
    "    for hp_exp_i in hp_exp:\n",
    "\n",
    "        # Wait until training slot is free\n",
    "        slot_free = False\n",
    "        while slot_free == False:\n",
    "            if count_models_training() < training_slots:\n",
    "                slot_free = True\n",
    "            else:\n",
    "                # wait 5 minutes before checking for free slot again\n",
    "                time.sleep(5*60)\n",
    "\n",
    "        model_creation_successful = False\n",
    "        while model_creation_successful == False:\n",
    "            try:\n",
    "                # Start training of model with hyperparams hp_exp_i\n",
    "                cloneModel(clonefrom=clone_from_model,\n",
    "                           modelname=clone_from_model+\"-clone-\" +\n",
    "                           chr(start_naming_with_int),\n",
    "                           description=str(hp_exp_i)[1:-1],\n",
    "                           track=track,\n",
    "                           hyperparams=hp_exp_i,\n",
    "                           maxtime=maxtime_per_training,\n",
    "                           rfchanges=None,\n",
    "                           testmode=False)\n",
    "            except:\n",
    "                # If model creation failed, print message and wait for 5 minutes\n",
    "                current_version = chr(start_naming_with_int)\n",
    "                now = datetime.datetime.now()\n",
    "                now_h = now.hour\n",
    "                now_m = now.minute\n",
    "                print(\n",
    "                    f\"{now_h}:{now_m} Model creation of {current_version} failed. Trying again in 5 minutes\")\n",
    "                # Try logging in to AWS again, in case user was logged out\n",
    "                try:\n",
    "                    awsLogin()\n",
    "                except:\n",
    "                    pass\n",
    "                time.sleep(5*60)\n",
    "            else:\n",
    "                time.sleep(5)\n",
    "                # If not automatically transferred to new model url, print error message\n",
    "                modelname_check = clone_from_model + \\\n",
    "                    \"-clone-\" + chr(start_naming_with_int)\n",
    "                if browser.current_url != (\"https://console.aws.amazon.com/deepracer/home?region=us-east-1#model/%s\" % modelname_check):\n",
    "                    print(f\"Model creation of {modelname_check} may have failed.\", end=' ')\n",
    "                    print(\"Check in the console if model was created and create it manually if needed\")\n",
    "\n",
    "                # If model creation was successful, escape the while loop\n",
    "                model_creation_successful = True\n",
    "                # Increasing naming number\n",
    "                start_naming_with_int += 1\n",
    "    \n",
    "    \n",
    "def submit_to_spain(modelname):\n",
    "\n",
    "    browser.get(\n",
    "        \"https://console.aws.amazon.com/deepracer/home?region=us-east-1#model/%s\" % modelname)\n",
    "    browser.refresh()\n",
    "    time.sleep(5)\n",
    "    \n",
    "    submitToRaceButton = browser.find_element_by_xpath(\n",
    "            '//button[@type=\"submit\"]/*[text()=\"Submit to virtual race\"]')\n",
    "    submitToRaceButton.click()\n",
    "    \n",
    "    submitModelButton = browser.find_element_by_xpath(\n",
    "            '//button[@type=\"submit\"]/*[text()=\"Submit model\"]')\n",
    "    submitModelButton.click()\n",
    "    \n",
    "    # Sometimes, pressing the submit button will not trigger a submit\n",
    "    # Therefore, just retry 5 times\n",
    "    re_press_submit = 5\n",
    "    while re_press_submit > 0:\n",
    "        try:\n",
    "            submitModelButton.click()\n",
    "            re_press_submit -= 1\n",
    "            time.sleep(2)\n",
    "        except:\n",
    "            # If click failed, means that submit was successful and we got re-routed to Event starting screen\n",
    "            re_press_submit = 0\n",
    "\n",
    "    time.sleep(15)\n",
    "\n",
    "    print(f\"{datetime.datetime.now()} Submitted model {modelname} to F1 Race\")\n",
    "    \n",
    "    \n",
    "def submit_to_spain_multiple(modelname, repeat_hours=9):\n",
    "\n",
    "    # Calculate when to stop\n",
    "    datetime_stop = datetime.datetime.now() + datetime.timedelta(hours=repeat_hours)\n",
    "\n",
    "    # Count number of submits\n",
    "    count_submits = 0\n",
    "    count_fails = 0\n",
    "\n",
    "    # Repeat loop until time is up\n",
    "    while datetime.datetime.now() < datetime_stop:\n",
    "        try:\n",
    "            # Submit model to summit\n",
    "            submit_to_spain(modelname=modelname)\n",
    "            # Wait for 10 minutes before attempting submit again\n",
    "            time.sleep(10*60)\n",
    "        except:\n",
    "            # If failed to submit, wait for 2 minutes and try again\n",
    "            count_fails += 1\n",
    "            time.sleep(2*60)\n",
    "            # If failed 5 times, try to log back in\n",
    "            if count_fails >= 10:\n",
    "                awsLogin()\n",
    "        else:\n",
    "            # If there was no error, increase counter by 1\n",
    "            count_submits += 1\n",
    "            count_fails = 0\n",
    "\n",
    "    # Print final submit count\n",
    "    print(f\"Submitted number of models to the race: {count_submits}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EXECUTE CODE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log in to AWS\n",
    "- This cell should always be executed one time in the beginning\n",
    "- Important: The directory of this notebook should have a AWS_credentials.txt file, which has 3 lines: AWS_id, username, and password. This will not work with an AWS root account"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Quits past browser instance\n",
    "quitBrowser()\n",
    "\n",
    "# Creates new browser instance\n",
    "browser = newBrowser()\n",
    "\n",
    "# Log in to AWS using AWS_credentials.txt\n",
    "awsLogin()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submit model to race multiple times"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### F1 Event (Spain track)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Submit the model to the summit race for multiple hours\n",
    "submit_to_spain_multiple(modelname=\"MODELNAME\", repeat_hours=12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clone model and run experiments\n",
    "This cell runs experiments on hyperparameters. Be aware, that this could take several hours to execute. Best to let it run over night.\n",
    "1. Define hyperparameters that should be tested.\n",
    "2. clone_from_model: Name of model, which should be cloned\n",
    "3. training_slots: Maximum number of parallel training sessions that AWS allows (usually 4, but only 2 in May 2020)\n",
    "4. number_of_experiments: As many combinations of hyperparameters are possible, only a hand full should be randomly selected and tested\n",
    "5. start_naming_with: Defines with which character the clones should start being named. Be careful to start with a character that does not exist yet, otherwise AWS will give an error when trying to create the model\n",
    "6. track: Track name (e.g. \"reInvent2019\" or \"Spain\")\n",
    "7. maxtime_per_training: Define the number of minutes that each clone should be trained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All available hyperparameters\n",
    "# \"batchsize\": [32,64,128,256,512], \n",
    "# \"epochs\": [3 to 10], \n",
    "# \"learningrate\": [0.00000001 to 0.001], \n",
    "# \"entropy\": [0 to 1], \n",
    "# \"discount\": [0 to 1], \n",
    "# \"episodesUpdate\": [5 to 100]\n",
    "\n",
    "experiment = {\"batchsize\":[64,256], \n",
    "              \"epochs\":[5,10], \n",
    "              \"learningrate\":[0.0001,0.0002,0.0003], \n",
    "              \"entropy\":[0.005,0.01], \n",
    "              \"discount\":[0.9993], \n",
    "              \"episodesUpdate\":[20]}\n",
    "\n",
    "clone_hyperparams_experiment(clone_from_model=\"Cup2019-v14-clone-e\", \n",
    "                             hyperparams_experiment=experiment,\n",
    "                             training_slots=2, \n",
    "                             number_of_experiments=24, \n",
    "                             start_naming_with=\"a\", \n",
    "                             track=\"reInvent2019\", \n",
    "                             maxtime_per_training=180)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
